# __author__ = 'chenbinghui'
#!/usr/bin/python
# -*- coding: UTF-8 -*-

import scrapy
import logging
from turorial.items import IpAddressItem

class AgencySpider(scrapy.Spider):
    name = "Ipagency"
    start_urls = ['http://www.xicidaili.com/nn',]
    post_headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Content-Type": "application/x-www-form-urlencoded",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36",
        "Referer": "http://www.xicidaili.com/",
    }
    def start_requests(self):
        yield scrapy.Request(url='http://www.xicidaili.com/nn',headers=self.post_headers)
    def parse(self, response):
        lists = response.xpath('//table[@id="ip_list"]/tr[@class="odd"]')
        items=[]
        for iplist in lists:
            item = IpAddressItem()
            # item['ipaddress'] = iplist.xpath('//tr/td[2]/text()').extract_first()
            # item['port'] = iplist.xpath('//tr/td[3]/text()').extract_first()
            # item['ipaddress'] = iplist.css('td:nth-child(2)::text').extract_first()
            # item['port'] = iplist.css('td:nth-child(3)::text').extract_first()
            item['ipaddress'] = iplist.xpath('td[2]/text()').extract_first()
            item['port'] = iplist.xpath('td[3]/text()').extract_first()
            item['place'] = iplist.xpath('td[4]/a/text()').extract_first()
            item['life'] = iplist.xpath('td[9]/text()').extract_first()
            print('ipaddress:%s---port:%s----place:%s-----life:%s' %(item['ipaddress'],item['port'],item['place'],item['life']))
            items.append(item)
            yield item
        return items